package com.book.book.service;

import com.book.book.NovelSiteEnum;
import com.book.common.utils.NovelSpiderHttpGet;
import com.book.common.utils.NovelSpiderUtil;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.net.URL;
import java.util.Iterator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/***
 * @Author Lin
 * @Date Create By 22:31 2017/10/4
 * @Description
 */
public abstract class AbstractSpider {

    /**
     * @param url
     * @Author: Lin
     * @Description: 抓取指定网站的内容
     * @Date: 22:33 2017/10/4
     */
    protected String crawl(String url) throws Exception {
        try (CloseableHttpClient httpClient = HttpClientBuilder.create().build();
             CloseableHttpResponse httpResponse = httpClient.execute(new NovelSpiderHttpGet(url))) {

            String charset = AbstractSpider.getCharset(url);
            System.out.println("这个网页的编码为:"+charset);

            //String result = EntityUtils.toString(httpResponse.getEntity(),
            //        NovelSpiderUtil.getContext(NovelSiteEnum.getEnumByUrl(url)).get("charset")).trim();
            String result = EntityUtils.toString(httpResponse.getEntity(),charset);
            return result;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }

    }

    /**
     * 获得字符集
     */
    public static String getCharset (String siteurl) throws Exception{
        URL url = new URL(siteurl);
        Document doc = Jsoup.parse(url, 6*1000);
        Elements eles = doc.select("meta[http-equiv=Content-Type]");
        Iterator<Element> itor = eles.iterator();
        while (itor.hasNext())
            return AbstractSpider.matchCharset(itor.next().toString());
        return "gb2312";
    }

    /**
     * 获得页面字符
     */
    public static String matchCharset(String content) {
        String chs = "gb2312";
        Pattern p = Pattern.compile("(?<=charset=)(.+)(?=\")");
        Matcher m = p.matcher(content);
        if (m.find())
            return m.group();
        return chs;
    }
}
